{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Using TensorFlow backend.\n",
      "/home/fanyixing/.local/python3/lib/python3.6/importlib/_bootstrap.py:219: RuntimeWarning: numpy.dtype size changed, may indicate binary incompatibility. Expected 96, got 88\n",
      "  return f(*args, **kwds)\n",
      "/home/fanyixing/.local/python3/lib/python3.6/importlib/_bootstrap.py:219: RuntimeWarning: numpy.dtype size changed, may indicate binary incompatibility. Expected 96, got 88\n",
      "  return f(*args, **kwds)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "matchzoo version 2.1.0\n",
      "\n",
      "data loading ...\n",
      "data loaded as `train_pack_raw` `dev_pack_raw` `test_pack_raw`\n",
      "`ranking_task` initialized with metrics [normalized_discounted_cumulative_gain@3(0.0), normalized_discounted_cumulative_gain@5(0.0), mean_average_precision(0.0)]\n",
      "loading embedding ...\n",
      "embedding loaded as `glove_embedding`\n"
     ]
    }
   ],
   "source": [
    "%run init.ipynb"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Processing text_left with chain_transform of Tokenize => Lowercase => PuncRemoval: 100%|██████████| 2118/2118 [00:00<00:00, 3665.85it/s]\n",
      "Processing text_right with chain_transform of Tokenize => Lowercase => PuncRemoval: 100%|██████████| 18841/18841 [00:08<00:00, 2124.56it/s]\n",
      "Processing text_right with append: 100%|██████████| 18841/18841 [00:00<00:00, 312816.21it/s]\n",
      "Building FrequencyFilter from a datapack.: 100%|██████████| 18841/18841 [00:00<00:00, 58156.99it/s]\n",
      "Processing text_right with transform: 100%|██████████| 18841/18841 [00:00<00:00, 36334.58it/s]\n",
      "Processing text_left with extend: 100%|██████████| 2118/2118 [00:00<00:00, 344069.71it/s]\n",
      "Processing text_right with extend: 100%|██████████| 18841/18841 [00:00<00:00, 301782.94it/s]\n",
      "Building Vocabulary from a datapack.: 100%|██████████| 404415/404415 [00:00<00:00, 1562426.96it/s]\n",
      "Processing text_left with chain_transform of Tokenize => Lowercase => PuncRemoval: 100%|██████████| 2118/2118 [00:00<00:00, 3087.94it/s]\n",
      "Processing text_right with chain_transform of Tokenize => Lowercase => PuncRemoval: 100%|██████████| 18841/18841 [00:10<00:00, 1847.91it/s]\n",
      "Processing text_right with transform: 100%|██████████| 18841/18841 [00:00<00:00, 55576.41it/s]\n",
      "Processing text_left with transform: 100%|██████████| 2118/2118 [00:00<00:00, 37457.67it/s]\n",
      "Processing text_right with transform: 100%|██████████| 18841/18841 [00:00<00:00, 55490.83it/s]\n",
      "Processing length_left with len: 100%|██████████| 2118/2118 [00:00<00:00, 329263.75it/s]\n",
      "Processing length_right with len: 100%|██████████| 18841/18841 [00:00<00:00, 396878.61it/s]\n",
      "Processing text_left with transform: 100%|██████████| 2118/2118 [00:00<00:00, 38067.78it/s]\n",
      "Processing text_right with transform: 100%|██████████| 18841/18841 [00:00<00:00, 20157.65it/s]\n",
      "Processing text_left with chain_transform of Tokenize => Lowercase => PuncRemoval: 100%|██████████| 122/122 [00:00<00:00, 2859.98it/s]\n",
      "Processing text_right with chain_transform of Tokenize => Lowercase => PuncRemoval: 100%|██████████| 1115/1115 [00:00<00:00, 1982.49it/s]\n",
      "Processing text_right with transform: 100%|██████████| 1115/1115 [00:00<00:00, 38183.90it/s]\n",
      "Processing text_left with transform: 100%|██████████| 122/122 [00:00<00:00, 29319.03it/s]\n",
      "Processing text_right with transform: 100%|██████████| 1115/1115 [00:00<00:00, 43965.86it/s]\n",
      "Processing length_left with len: 100%|██████████| 122/122 [00:00<00:00, 89240.51it/s]\n",
      "Processing length_right with len: 100%|██████████| 1115/1115 [00:00<00:00, 260871.81it/s]\n",
      "Processing text_left with transform: 100%|██████████| 122/122 [00:00<00:00, 26225.15it/s]\n",
      "Processing text_right with transform: 100%|██████████| 1115/1115 [00:00<00:00, 25368.18it/s]\n",
      "Processing text_left with chain_transform of Tokenize => Lowercase => PuncRemoval: 100%|██████████| 237/237 [00:00<00:00, 1868.81it/s]\n",
      "Processing text_right with chain_transform of Tokenize => Lowercase => PuncRemoval: 100%|██████████| 2300/2300 [00:01<00:00, 1421.59it/s]\n",
      "Processing text_right with transform: 100%|██████████| 2300/2300 [00:00<00:00, 25433.49it/s]\n",
      "Processing text_left with transform: 100%|██████████| 237/237 [00:00<00:00, 30281.48it/s]\n",
      "Processing text_right with transform: 100%|██████████| 2300/2300 [00:00<00:00, 47353.48it/s]\n",
      "Processing length_left with len: 100%|██████████| 237/237 [00:00<00:00, 188088.94it/s]\n",
      "Processing length_right with len: 100%|██████████| 2300/2300 [00:00<00:00, 356197.59it/s]\n",
      "Processing text_left with transform: 100%|██████████| 237/237 [00:00<00:00, 19210.92it/s]\n",
      "Processing text_right with transform: 100%|██████████| 2300/2300 [00:00<00:00, 24293.37it/s]\n"
     ]
    }
   ],
   "source": [
    "preprocessor = mz.preprocessors.BasicPreprocessor(fixed_length_left=10, fixed_length_right=40, remove_stop_words=False)\n",
    "train_pack_processed = preprocessor.fit_transform(train_pack_raw)\n",
    "valid_pack_processed = preprocessor.transform(dev_pack_raw)\n",
    "test_pack_processed = preprocessor.transform(test_pack_raw)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'filter_unit': <matchzoo.preprocessors.units.frequency_filter.FrequencyFilter at 0x7fd9f4649be0>,\n",
       " 'vocab_unit': <matchzoo.preprocessors.units.vocabulary.Vocabulary at 0x7fd938f312b0>,\n",
       " 'vocab_size': 16674,\n",
       " 'embedding_input_dim': 16674,\n",
       " 'input_shapes': [(10,), (40,)]}"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "preprocessor.context"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "model = mz.models.KNRM()\n",
    "model.params.update(preprocessor.context)\n",
    "model.params['task'] = ranking_task\n",
    "model.params['embedding_output_dim'] = glove_embedding.output_dim\n",
    "model.params['embedding_trainable'] = True\n",
    "model.params['kernel_num'] = 21\n",
    "model.params['sigma'] = 0.1\n",
    "model.params['exact_sigma'] = 0.001\n",
    "model.params['optimizer'] = 'adadelta'\n",
    "model.build()\n",
    "model.compile()\n",
    "#model.backend.summary()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "embedding_matrix = glove_embedding.build_matrix(preprocessor.context['vocab_unit'].state['term_index'])\n",
    "model.load_embedding_matrix(embedding_matrix)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "pred_x, pred_y = test_pack_processed.unpack()\n",
    "evaluate = mz.callbacks.EvaluateAllMetrics(model, x=pred_x, y=pred_y, batch_size=len(pred_x))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "num batches: 255\n"
     ]
    }
   ],
   "source": [
    "train_generator = mz.DataGenerator(\n",
    "    train_pack_processed,\n",
    "    mode='pair',\n",
    "    num_dup=5,\n",
    "    num_neg=1,\n",
    "    batch_size=20\n",
    ")\n",
    "print('num batches:', len(train_generator))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 1/30\n",
      "255/255 [==============================] - 30s 117ms/step - loss: 1.0630\n",
      "Validation: normalized_discounted_cumulative_gain@3(0.0): 0.525888722537362 - normalized_discounted_cumulative_gain@5(0.0): 0.5969076581909297 - mean_average_precision(0.0): 0.5458421758049457\n",
      "Epoch 2/30\n",
      "255/255 [==============================] - 40s 158ms/step - loss: 0.4855\n",
      "Validation: normalized_discounted_cumulative_gain@3(0.0): 0.540316755776057 - normalized_discounted_cumulative_gain@5(0.0): 0.6183376741777723 - mean_average_precision(0.0): 0.5712611133729898\n",
      "Epoch 3/30\n",
      "255/255 [==============================] - 39s 155ms/step - loss: 0.3582\n",
      "Validation: normalized_discounted_cumulative_gain@3(0.0): 0.541616192577938 - normalized_discounted_cumulative_gain@5(0.0): 0.6153837509826408 - mean_average_precision(0.0): 0.567777172721793\n",
      "Epoch 4/30\n",
      "255/255 [==============================] - 40s 156ms/step - loss: 0.2887\n",
      "Validation: normalized_discounted_cumulative_gain@3(0.0): 0.5584765720489868 - normalized_discounted_cumulative_gain@5(0.0): 0.6194525103413756 - mean_average_precision(0.0): 0.5778673022579046\n",
      "Epoch 5/30\n",
      "255/255 [==============================] - 41s 162ms/step - loss: 0.2118\n",
      "Validation: normalized_discounted_cumulative_gain@3(0.0): 0.5528840743131206 - normalized_discounted_cumulative_gain@5(0.0): 0.6232796348158448 - mean_average_precision(0.0): 0.5763382056568952\n",
      "Epoch 6/30\n",
      "255/255 [==============================] - 41s 160ms/step - loss: 0.1721\n",
      "Validation: normalized_discounted_cumulative_gain@3(0.0): 0.5479498178842863 - normalized_discounted_cumulative_gain@5(0.0): 0.6080774335846967 - mean_average_precision(0.0): 0.5628146118070727\n",
      "Epoch 7/30\n",
      "255/255 [==============================] - 42s 163ms/step - loss: 0.1351\n",
      "Validation: normalized_discounted_cumulative_gain@3(0.0): 0.5448249760251662 - normalized_discounted_cumulative_gain@5(0.0): 0.6050334611320453 - mean_average_precision(0.0): 0.5539922350259576\n",
      "Epoch 8/30\n",
      "255/255 [==============================] - 41s 161ms/step - loss: 0.1056\n",
      "Validation: normalized_discounted_cumulative_gain@3(0.0): 0.5278279251987681 - normalized_discounted_cumulative_gain@5(0.0): 0.5926550462383007 - mean_average_precision(0.0): 0.546292777345115\n",
      "Epoch 9/30\n",
      "255/255 [==============================] - 40s 155ms/step - loss: 0.0843\n",
      "Validation: normalized_discounted_cumulative_gain@3(0.0): 0.5260762848714361 - normalized_discounted_cumulative_gain@5(0.0): 0.5997466236266378 - mean_average_precision(0.0): 0.5463433489984417\n",
      "Epoch 10/30\n",
      "255/255 [==============================] - 39s 151ms/step - loss: 0.0641\n",
      "Validation: normalized_discounted_cumulative_gain@3(0.0): 0.53545626407713 - normalized_discounted_cumulative_gain@5(0.0): 0.5911736172377866 - mean_average_precision(0.0): 0.5511742297719537\n",
      "Epoch 11/30\n",
      "255/255 [==============================] - 39s 154ms/step - loss: 0.0506\n",
      "Validation: normalized_discounted_cumulative_gain@3(0.0): 0.5281313957863254 - normalized_discounted_cumulative_gain@5(0.0): 0.5909946137693491 - mean_average_precision(0.0): 0.5455621168460892\n",
      "Epoch 12/30\n",
      "255/255 [==============================] - 42s 163ms/step - loss: 0.0411\n",
      "Validation: normalized_discounted_cumulative_gain@3(0.0): 0.5295249224151654 - normalized_discounted_cumulative_gain@5(0.0): 0.5882240550416985 - mean_average_precision(0.0): 0.544818346189001\n",
      "Epoch 13/30\n",
      "255/255 [==============================] - 41s 162ms/step - loss: 0.0356\n",
      "Validation: normalized_discounted_cumulative_gain@3(0.0): 0.5332953580728098 - normalized_discounted_cumulative_gain@5(0.0): 0.5923864923271813 - mean_average_precision(0.0): 0.5449250708396572\n",
      "Epoch 14/30\n",
      "255/255 [==============================] - 42s 163ms/step - loss: 0.0275\n",
      "Validation: normalized_discounted_cumulative_gain@3(0.0): 0.5344059653683887 - normalized_discounted_cumulative_gain@5(0.0): 0.5852285609827376 - mean_average_precision(0.0): 0.5457076553713109\n",
      "Epoch 15/30\n",
      "255/255 [==============================] - 42s 163ms/step - loss: 0.0224\n",
      "Validation: normalized_discounted_cumulative_gain@3(0.0): 0.5326101057106915 - normalized_discounted_cumulative_gain@5(0.0): 0.5896572847849786 - mean_average_precision(0.0): 0.5461713501388032\n",
      "Epoch 16/30\n",
      "255/255 [==============================] - 40s 155ms/step - loss: 0.0189\n",
      "Validation: normalized_discounted_cumulative_gain@3(0.0): 0.5209623555446414 - normalized_discounted_cumulative_gain@5(0.0): 0.5907051989832439 - mean_average_precision(0.0): 0.544006088275105\n",
      "Epoch 17/30\n",
      "255/255 [==============================] - 39s 153ms/step - loss: 0.0154\n",
      "Validation: normalized_discounted_cumulative_gain@3(0.0): 0.5270528188502702 - normalized_discounted_cumulative_gain@5(0.0): 0.5929241490222487 - mean_average_precision(0.0): 0.5458721064851292\n",
      "Epoch 18/30\n",
      "255/255 [==============================] - 39s 152ms/step - loss: 0.0134\n",
      "Validation: normalized_discounted_cumulative_gain@3(0.0): 0.52963998719156 - normalized_discounted_cumulative_gain@5(0.0): 0.59748806088061 - mean_average_precision(0.0): 0.5504225606330331\n",
      "Epoch 19/30\n",
      "255/255 [==============================] - 41s 161ms/step - loss: 0.0129\n",
      "Validation: normalized_discounted_cumulative_gain@3(0.0): 0.5173295480303289 - normalized_discounted_cumulative_gain@5(0.0): 0.5890029693924619 - mean_average_precision(0.0): 0.5393698842857736\n",
      "Epoch 20/30\n",
      "255/255 [==============================] - 42s 164ms/step - loss: 0.0136\n",
      "Validation: normalized_discounted_cumulative_gain@3(0.0): 0.5319324815526434 - normalized_discounted_cumulative_gain@5(0.0): 0.599965023036905 - mean_average_precision(0.0): 0.550039654928675\n",
      "Epoch 21/30\n",
      "255/255 [==============================] - 42s 163ms/step - loss: 0.0108\n",
      "Validation: normalized_discounted_cumulative_gain@3(0.0): 0.5311517097282423 - normalized_discounted_cumulative_gain@5(0.0): 0.5943106559500474 - mean_average_precision(0.0): 0.5487263041947172\n",
      "Epoch 22/30\n",
      "255/255 [==============================] - 42s 165ms/step - loss: 0.0093\n",
      "Validation: normalized_discounted_cumulative_gain@3(0.0): 0.5340421861939937 - normalized_discounted_cumulative_gain@5(0.0): 0.6020890805193887 - mean_average_precision(0.0): 0.5541712982869859\n",
      "Epoch 23/30\n",
      "255/255 [==============================] - 40s 157ms/step - loss: 0.0063\n",
      "Validation: normalized_discounted_cumulative_gain@3(0.0): 0.5245100179927665 - normalized_discounted_cumulative_gain@5(0.0): 0.5896176258663225 - mean_average_precision(0.0): 0.5427922534821856\n",
      "Epoch 24/30\n",
      "255/255 [==============================] - 40s 157ms/step - loss: 0.00841s - loss\n",
      "Validation: normalized_discounted_cumulative_gain@3(0.0): 0.528425907303436 - normalized_discounted_cumulative_gain@5(0.0): 0.5952097664878763 - mean_average_precision(0.0): 0.5498493585406231\n",
      "Epoch 25/30\n",
      "255/255 [==============================] - 41s 160ms/step - loss: 0.0071\n",
      "Validation: normalized_discounted_cumulative_gain@3(0.0): 0.5303218965746611 - normalized_discounted_cumulative_gain@5(0.0): 0.5948210455548147 - mean_average_precision(0.0): 0.5505741686543946\n",
      "Epoch 26/30\n",
      "255/255 [==============================] - 41s 160ms/step - loss: 0.0065\n",
      "Validation: normalized_discounted_cumulative_gain@3(0.0): 0.5223105087495103 - normalized_discounted_cumulative_gain@5(0.0): 0.5901051330822427 - mean_average_precision(0.0): 0.5426582852986084\n",
      "Epoch 27/30\n",
      "255/255 [==============================] - 42s 164ms/step - loss: 0.0045\n",
      "Validation: normalized_discounted_cumulative_gain@3(0.0): 0.5259626991148388 - normalized_discounted_cumulative_gain@5(0.0): 0.5922912075108971 - mean_average_precision(0.0): 0.5439769673667663\n",
      "Epoch 28/30\n",
      "255/255 [==============================] - 42s 164ms/step - loss: 0.0039\n",
      "Validation: normalized_discounted_cumulative_gain@3(0.0): 0.5326394392626637 - normalized_discounted_cumulative_gain@5(0.0): 0.5991254533953382 - mean_average_precision(0.0): 0.5509986930248422\n",
      "Epoch 29/30\n",
      "255/255 [==============================] - 34s 134ms/step - loss: 0.0037\n",
      "Validation: normalized_discounted_cumulative_gain@3(0.0): 0.5250020425449076 - normalized_discounted_cumulative_gain@5(0.0): 0.5952453236394062 - mean_average_precision(0.0): 0.5498564190858788\n",
      "Epoch 30/30\n",
      "255/255 [==============================] - 35s 138ms/step - loss: 0.0046\n",
      "Validation: normalized_discounted_cumulative_gain@3(0.0): 0.5309288871342496 - normalized_discounted_cumulative_gain@5(0.0): 0.5931766459815412 - mean_average_precision(0.0): 0.5474981722703828\n"
     ]
    }
   ],
   "source": [
    "history = model.fit_generator(train_generator, epochs=30, callbacks=[evaluate], workers=30, use_multiprocessing=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
